Read in MoTR Data
rate = 160
file_prefix = "/Users/cui/Desktop/motr_provo_160/"
fnames = list.files(path=file_prefix)
df = data.frame()
for (f in fnames) {
temp = read.csv(paste0(file_prefix, "/", f)) %>%
mutate(subj = str_remove(f, "_reading_measures.csv")) %>%
rename(go_past_time = go_pass_time)
df = rbind(df, temp)
}
filter_df = df %>%
group_by(para_nr, subj) %>%
summarise(correct = if_else(unique(correctness) == 1, 1, 0)) %>%
ungroup() %>%
drop_na() %>%
group_by(subj) %>%
summarise(p_correct = mean(correct)) %>%
ungroup() %>%
mutate(p_correct = round(p_correct, digits = 2))
`summarise()` has grouped output by 'para_nr'. You can override using the `.groups` argument.
filter_df = filter_df %>%
filter(p_correct < 0.8)
# View(filter_df)
## reader_3:0.70, reader_60:0.79, reader_76:0.72 , reader_256:0.71 , reader_262:0.57
raw_df = df %>%
mutate(word = str_trim(word)) %>%
mutate(subj = str_remove(subj, "reader_")) %>%
mutate(subj = as.integer(subj)) %>%
filter(! subj %in% c(3, 60, 76, 256, 262)) %>%
# See below for filtering out reading measures that are super long
dplyr::select(expr_id, cond_id, para_nr, word, word_nr, first_duration, total_duration, gaze_duration, go_past_time, FPReg, subj) %>%
drop_na()
raw_df
# Average across subjects
motr_agg_df = raw_df %>%
gather(metric, value, 6:10) %>%
group_by(para_nr, word_nr, word, metric) %>%
# for non-binary measures, filter out that are super long
mutate(outlier = if_else(metric != "FPReg" & value > (mean(value) + 3 * sd(value) ), T, F)) %>%
filter(outlier == F) %>%
# # Filter out words with a reading-time of zero
# mutate(zero = if_else(metric != "FPReg" & value == 0, T, F)) %>%
# filter(zero == F) %>%
drop_na() %>%
summarise(value = mean(value),
nsubj = length(unique(subj))) %>%
ungroup() %>%
arrange(para_nr, word_nr) %>%
rename(
text_id = para_nr,
word_text_idx = word_nr,
motr_value = value
)
`summarise()` has grouped output by 'para_nr', 'word_nr', 'word'. You can override using the `.groups` argument.
motr_agg_df
NA
Comparison to Provo
# Read in Provo surprisal, frequency and length data
provo_modeling_df = read.csv("/Users/cui/Documents/ETH/MoTR/pipeline/ancillary_data/provo_df.csv") %>%
dplyr::select(text_id, sent_id, trigger_idx, word, freq, surp, len) %>%
rename(word_idx = trigger_idx)
provo_modeling_df
NA
# Read in Provo eyetracking data
provo_raw_df = read.csv("/Users/cui/Documents/ETH/MoTR/pipeline/ancillary_data/provo_eyetracking.csv")
provo_eyetracking_df = provo_raw_df %>%
dplyr::select(Participant_ID, Text_ID, Sentence_Number, Word_In_Sentence_Number, Word, Word_Number, IA_FIRST_FIX_PROGRESSIVE, IA_FIRST_RUN_DWELL_TIME, IA_DWELL_TIME, IA_REGRESSION_PATH_DURATION, IA_REGRESSION_OUT) %>%
rename( #first_duration = IA_FIRST_FIXATION_DURATION,
gaze_duration = IA_FIRST_RUN_DWELL_TIME,
total_duration = IA_DWELL_TIME,
go_past_time = IA_REGRESSION_PATH_DURATION,
subj = Participant_ID,
text_id = Text_ID,
sent_id = Sentence_Number,
word_idx = Word_In_Sentence_Number,
word_text_idx = Word_Number, # IA_ID?
word = Word,
FPReg = IA_REGRESSION_OUT,
ff_progressive = IA_FIRST_FIX_PROGRESSIVE) %>%
mutate(first_duration = gaze_duration) %>%
mutate(gaze_duration = ifelse(ff_progressive == 0, 0, gaze_duration),
go_past_time = ifelse(ff_progressive == 0, 0, go_past_time)) %>%
dplyr::select(-ff_progressive) %>%
gather(metric, value, 7:11) %>%
mutate(value = if_else(is.na(value), as.integer(0), as.integer(value))) %>%
drop_na() %>%
mutate(word = str_trim(word)) %>%
mutate(subj = str_remove(subj, "Sub")) %>%
mutate(subj = as.integer(subj)) %>%
group_by(text_id, word_text_idx, sent_id, word_idx, word, metric) %>%
mutate(outlier = if_else(metric != "FPReg" & value > (mean(value) + 3 * sd(value) ), T, F)) %>%
filter(outlier == F) %>%
ungroup() #%>%
# # Filter out words with a reading-time of zero
# mutate(zero = if_else(metric != "FPReg" & value == 0,T, F)) %>%
# filter(zero == F)
# Aggregate cross-participant data for all subjects
provo_eyetracking_agg_df = provo_eyetracking_df %>%
group_by(text_id, word_text_idx, sent_id, word_idx, word, metric) %>%
summarise(value = mean(value), .groups = 'drop')
# Split the eyetracking data in two by subjects to see how well it correlates with itself
provo_eyetracking_subj1_df = provo_eyetracking_df %>%
filter(subj <= 42) %>%
group_by(text_id, word_text_idx, sent_id, word_idx, word, metric) %>%
summarise(value = mean(value)) %>%
ungroup() %>%
rename(value_1 = value) %>%
dplyr::select(-sent_id, -word_idx)
`summarise()` has grouped output by 'text_id', 'word_text_idx', 'sent_id', 'word_idx', 'word'. You can override using the `.groups` argument.
provo_eyetracking_subj2_df = provo_eyetracking_df %>%
filter(subj > 42) %>%
group_by(text_id, word_text_idx, sent_id, word_idx, word, metric) %>%
summarise(value = mean(value)) %>%
ungroup() %>%
rename(value_2 = value)%>%
dplyr::select(-sent_id, -word_idx)
`summarise()` has grouped output by 'text_id', 'word_text_idx', 'sent_id', 'word_idx', 'word'. You can override using the `.groups` argument.
provo_eyetr_grouped_df = merge(provo_eyetracking_subj2_df, provo_eyetracking_subj1_df, by=c("text_id", "word_text_idx", "metric")) %>%
filter(word.x == word.y) %>%
dplyr::select(-word.x, -word.y) %>%
group_by(metric) %>%
mutate(motr_outlier = if_else(metric != "FPReg" & value_1 > (mean(value_1) + 3 * sd(value_1) ), T, F)) %>%
filter(motr_outlier == F) %>%
mutate(eyetr_outlier = if_else(metric != "FPReg" & value_2 > (mean(value_2) + 3 * sd(value_2) ), T, F)) %>%
filter(eyetr_outlier == F) %>%
ungroup() %>%
gather(measure, value, c("value_1", "value_2")) %>%
dplyr::select(-motr_outlier, -eyetr_outlier)
provo_df = merge(provo_eyetracking_agg_df, provo_modeling_df, by=c("text_id", "sent_id", "word_idx")) %>%
mutate(word_text_idx = as.integer(word_text_idx - 1)) %>%
arrange(text_id, sent_id, word_idx) %>%
rename(eyetr_value = value)
provo_df = merge(provo_df, motr_agg_df, by=c("text_id", "word_text_idx", "metric")) %>%
arrange(text_id, sent_id, word_idx) %>%
# almost all the word.x != word.y is because of normalization problem, so we can keep them, instead, deleting some special cases
filter(!(text_id == 13 & word_text_idx >= 20 & word_text_idx <= 52)) %>%
filter(!(text_id == 3 & word_text_idx >= 46 & word_text_idx <= 57)) %>%
# # filter(word.y == word) %>%
dplyr::select(-word.x, -word.y) %>%
group_by(metric) %>%
mutate(motr_outlier = if_else(metric != "FPReg" & motr_value > (mean(motr_value) + 3 * sd(motr_value) ), T, F)) %>%
filter(motr_outlier == F) %>%
mutate(eyetr_outlier = if_else(metric != "FPReg" & eyetr_value > (mean(eyetr_value) + 3 * sd(eyetr_value) ), T, F)) %>%
filter(eyetr_outlier == F) %>%
ungroup() %>%
gather(measure, value, c("eyetr_value", "motr_value")) %>%
dplyr::select(-motr_outlier, -eyetr_outlier)
provo_df %>%
mutate(measure = if_else(measure == "eyetr_value", "Eyetracking Value", "MoTR Value")) %>%
filter(metric != "FPReg") %>%
ggplot(aes(x = value, color=metric)) +
geom_density() +
facet_wrap(.~measure, scales="free_y") +
xlab("Reading Time (ms)")

# ggsave("../visualization/density.png", device = "png", width = 6, height = 2.5)
print("Gaze Duration")
[1] "Gaze Duration"
cor_df = provo_df %>% filter(metric == "gaze_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$motr_value)$estimate)
cor
0.7874
cor_df = provo_eyetr_grouped_df %>% filter(metric == "gaze_duration") %>% spread(measure, value)
print(cor.test(cor_df$value_1, cor_df$value_2)$estimate)
cor
0.9168
print("First Duration")
[1] "First Duration"
gd_df = provo_df %>% filter(metric == "first_duration") %>% spread(measure, value)
print(cor.test(gd_df$eyetr_value, gd_df$motr_value)$estimate)
cor
0.7815
cor_df = provo_eyetr_grouped_df %>% filter(metric == "first_duration") %>% spread(measure, value)
print(cor.test(cor_df$value_1, cor_df$value_2)$estimate)
cor
0.9201
print("Go Past Time")
[1] "Go Past Time"
gd_df = provo_df %>% filter(metric == "go_past_time") %>% spread(measure, value)
print(cor.test(gd_df$eyetr_value, gd_df$motr_value)$estimate)
cor
0.7292
cor_df = provo_eyetr_grouped_df %>% filter(metric == "go_past_time") %>% spread(measure, value)
print(cor.test(cor_df$value_1, cor_df$value_2)$estimate)
cor
0.9133
print("Total Duration")
[1] "Total Duration"
gd_df = provo_df %>% filter(metric == "total_duration") %>% spread(measure, value)
print(cor.test(gd_df$eyetr_value, gd_df$motr_value)$estimate)
cor
0.7601
cor_df = provo_eyetr_grouped_df %>% filter(metric == "total_duration") %>% spread(measure, value)
print(cor.test(cor_df$value_1, cor_df$value_2)$estimate)
cor
0.9275
print("Regression")
[1] "Regression"
reg_df = provo_df %>% filter(metric == "FPReg") %>% spread(measure, value)
print(cor.test(reg_df$eyetr_value, reg_df$motr_value)$estimate)
cor
0.3259
cor_df = provo_eyetr_grouped_df %>% filter(metric == "FPReg") %>% group_by(text_id, metric, measure) %>%
summarize(value = mean(value, na.rm = TRUE), .groups = 'drop') %>% spread(measure, value)
print(cor.test(cor_df$value_1, cor_df$value_2)$estimate)
cor
0.6807
provo_df %>%
filter(metric != "FPReg") %>%
spread(measure, value) %>%
ggplot(aes(x = motr_value, y=eyetr_value)) +
geom_point(alpha = 0.05) +
geom_abline(slope=1, intercept=0, color = "black") +
#stat_summary_bin(bins=100, fun.data = "mean_cl_boot", size = 0.05) +
facet_wrap(.~metric, scales = "free", nrow = 1) +
coord_cartesian(ylim=c(0, 500), xlim=c(0, 500)) +
geom_smooth()

# ggsave("../visualization/metric_cor.png", device = "png", width = 6, height = 2.5)
Correlations to Word-Level Statistical Properties
print("Gaze Duration")
[1] "Gaze Duration"
print("Len")
[1] "Len"
cor_df = provo_df %>% filter(metric == "gaze_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$len)$estimate)
cor
0.8597
print(cor.test(cor_df$motr_value, cor_df$len)$estimate)
cor
0.8644
print("Freq")
[1] "Freq"
cor_df = provo_df %>% filter(metric == "gaze_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$freq)$estimate)
cor
-0.8069
print(cor.test(cor_df$motr_value, cor_df$freq)$estimate)
cor
-0.7454
print("Surp")
[1] "Surp"
cor_df = provo_df %>% filter(metric == "gaze_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$surp)$estimate)
cor
0.5683
print(cor.test(cor_df$motr_value, cor_df$surp)$estimate)
cor
0.4978
print("Total Duration")
[1] "Total Duration"
print("Len")
[1] "Len"
cor_df = provo_df %>% filter(metric == "total_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$len)$estimate)
cor
0.825
print(cor.test(cor_df$motr_value, cor_df$len)$estimate)
cor
0.838
print("Freq")
[1] "Freq"
cor_df = provo_df %>% filter(metric == "total_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$freq)$estimate)
cor
-0.781
print(cor.test(cor_df$motr_value, cor_df$freq)$estimate)
cor
-0.7258
print("Surp")
[1] "Surp"
cor_df = provo_df %>% filter(metric == "total_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$surp)$estimate)
cor
0.5931
print(cor.test(cor_df$motr_value, cor_df$surp)$estimate)
cor
0.5104
print("First Duration")
[1] "First Duration"
print("Len")
[1] "Len"
cor_df = provo_df %>% filter(metric == "first_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$len)$estimate)
cor
0.8444
print(cor.test(cor_df$motr_value, cor_df$len)$estimate)
cor
0.8598
print("Freq")
[1] "Freq"
cor_df = provo_df %>% filter(metric == "first_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$freq)$estimate)
cor
-0.8016
print(cor.test(cor_df$motr_value, cor_df$freq)$estimate)
cor
-0.7453
print("Surp")
[1] "Surp"
cor_df = provo_df %>% filter(metric == "first_duration") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$surp)$estimate)
cor
0.5852
print(cor.test(cor_df$motr_value, cor_df$surp)$estimate)
cor
0.507
print("Go Past Time")
[1] "Go Past Time"
print("Len")
[1] "Len"
cor_df = provo_df %>% filter(metric == "go_past_time") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$len)$estimate)
cor
0.8063
print(cor.test(cor_df$motr_value, cor_df$len)$estimate)
cor
0.8124
print("Freq")
[1] "Freq"
cor_df = provo_df %>% filter(metric == "go_past_time") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$freq)$estimate)
cor
-0.7516
print(cor.test(cor_df$motr_value, cor_df$freq)$estimate)
cor
-0.702
print("Surp")
[1] "Surp"
cor_df = provo_df %>% filter(metric == "go_past_time") %>% spread(measure, value)
print(cor.test(cor_df$eyetr_value, cor_df$surp)$estimate)
cor
0.5476
print(cor.test(cor_df$motr_value, cor_df$surp)$estimate)
cor
0.4893
provo_df %>%
gather(word_prop, word_prop_val, c("freq", "len", "surp")) %>%
mutate(measure = if_else(measure == "eyetr_value", "Eyetracking Value", "MoTR Value")) %>%
mutate(word_prop = case_when(
word_prop == "freq" ~ "Frequency",
word_prop == "len" ~ "Length",
word_prop == "surp" ~ "Surprisal"
)) %>%
filter(metric == "gaze_duration") %>%
ggplot(aes(x = value, y=word_prop_val, color = measure)) +
geom_point(alpha = 0.1) +
facet_wrap(measure~word_prop, scales="free", strip.position = "right") +
geom_smooth(color = "grey") +
xlab("Reading Measure") +
theme(
legend.position = "none",
strip.placement = "outside"
)

# ggsave("../visualization/word_prop_comps.png", device = "png", width = 6, height = 3)
provo_df %>%
ggplot(aes(x = value, y=freq, color=metric)) +
geom_point(alpha = 0.1) +
facet_grid(metric~measure, scales="free") +
geom_smooth()

provo_df %>%
ggplot(aes(x = value, y=surp, color=metric)) +
geom_point(alpha = 0.2) +
facet_grid(metric~measure, scales="free") +
geom_smooth()

Shape of surprisal / RT relationship
for current word:
fit_gam_inner = function(bootstrap_sample, mean_predictors) {
df = bootstrap_sample$data
weights = tabulate(as.integer(bootstrap_sample), nrow(df))
m = gam(psychometric ~ s(surp, bs = 'cr', k = 6) + s(prev_surp, bs = 'cr', k = 6) + te(freq, len, bs = 'cr') + te(prev_freq, prev_len, bs = 'cr'), data = df, weights = weights)
terms_to_predict = c("s(surp)", "s(prev_surp)")
newdata = data.frame(surp=seq(0,20,by=0.1), prev_surp=mean_predictors$surp,
#surp=mean_predictors$surp, prev_surp=seq(0,20,by=0.1),
freq=mean_predictors$freq, prev_freq=mean_predictors$freq,
len=mean_predictors$len, prev_len=mean_predictors$len)
# Returns a matrix N_samples * N_terms.
per_term_predictions = predict(m, newdata=newdata, terms=terms_to_predict, type="terms")
# Additive model -- sum across predictor response contributions (matrix columns).
predictions = rowSums(per_term_predictions)
return(newdata %>% mutate(y=predictions))
}
fit_gam = function(df, mean_predictors, alpha=0.05) {
# Bootstrap-resample data
boot_models = df %>% bootstraps(times=10) %>%
# Fit a GAM and get predictions for each sample
mutate(smoothed=map(splits, fit_gam_inner, mean_predictors=mean_predictors))
# Extract mean and 5% and 95% percentile y-values for each surprisal value
result = boot_models %>%
unnest(smoothed) %>%
dplyr::select(surp, y) %>%
#dplyr::select(prev_surp, y) %>%
group_by(surp) %>%
#group_by(prev_surp) %>%
summarise(y_lower=quantile(y, alpha / 2),
y_upper=quantile(y, 1 - alpha / 2),
y=mean(y)) %>%
ungroup()
return (result)
}
gam_modeling_df = provo_df %>%
spread(measure, value) %>%
# mutate(len = nchar(word)) %>% # len has already exists, but do not count punct into len.
group_by(metric, text_id) %>%
arrange(word_text_idx) %>%
mutate(prev_surp = lag(surp),
prev_freq = lag(freq),
prev_len = lag(len),
prev_eyetr_value = lag(eyetr_value)) %>%
ungroup() %>%
drop_na() %>%
rename(psychometric = motr_value)
smooths_df = data.frame()
metrics = c("gaze_duration", "total_duration", "go_past_time", "first_duration")
for (m in metrics) {
print(paste0("Fitting model for ", m))
dummy_df = gam_modeling_df %>% filter(metric == m)
mean_predictors = dummy_df %>% summarise(surp = mean(surp), len = mean(len), freq = mean(freq))
smooths = dummy_df %>% fit_gam(., mean_predictors)
#Fix 0 surprisal = 0 ms
gam_smooths = smooths %>% mutate(delta = 0 - y[1], y=y + delta, y_lower= y_lower + delta, y_upper=y_upper + delta)
smooths_df = rbind(smooths_df, gam_smooths %>% mutate(psychometric = m))
}
[1] "Fitting model for gaze_duration"
[1] "Fitting model for total_duration"
[1] "Fitting model for go_past_time"
[1] "Fitting model for first_duration"
Get Density Data
get_d_points = function(df) {
x = density(df$surp)$x
y = density(df$surp)$y
return(data.frame(x, y))
}
density_data = data.frame()
for(m in c("gaze_duration", "total_duration", "go_past_time", "first_duration")) {
dummy_df = provo_df %>% filter(metric == m) %>%
do({get_d_points(.)}) %>%
filter(x>0, x<20)
density_data = rbind(density_data, dummy_df %>% mutate(metric=m))
}
# Surprisal curves
ggplot() +
# Density Data
annotate("rect", xmin=0, xmax=20, ymin=-20,ymax=-10, fill="#f4f4f4", color="grey", alpha=1, size = 0) +
geom_line(data = density_data, aes(x=x, y=y*50 - 18), color="#aaaaaa", size = 0.4) +
# Surrp / Rt data
#geom_line(data = smooths_df, aes(x=prev_surp, y=y, color = psychometric), size=0.7) +
geom_line(data = smooths_df, aes(x=surp, y=y, color = psychometric), size=0.7) +
#geom_ribbon(data = smooths_df, aes(x=prev_surp, ymin=y_lower, ymax=y_upper, fill = psychometric), alpha=0.3, size=0.5) +
geom_ribbon(data = smooths_df, aes(x=surp, ymin=y_lower, ymax=y_upper, fill = psychometric), alpha=0.3, size=0.5) +
scale_x_continuous(labels=c(0, 10, 20), breaks=c(0, 10, 20), minor_breaks = NULL) +
facet_wrap(psychometric~., nrow = 1) +
ylab("Slowdown due to Surprisal (ms)") +
xlab("Surprisal of Word") +
ggtitle("MoTR Times and Current Word Surprisal")

theme(
legend.position = "none",
panel.grid.minor = element_blank()
)
List of 2
$ legend.position : chr "none"
$ panel.grid.minor: list()
..- attr(*, "class")= chr [1:2] "element_blank" "element"
- attr(*, "class")= chr [1:2] "theme" "gg"
- attr(*, "complete")= logi FALSE
- attr(*, "validate")= logi TRUE
# ggsave("../visualization/surprisal_rt_link.png", device = "png", width = 6, height = 2.5)
for previous word:
fit_gam_inner_2 = function(bootstrap_sample, mean_predictors) {
df = bootstrap_sample$data
weights = tabulate(as.integer(bootstrap_sample), nrow(df))
m = gam(psychometric ~ s(surp, bs = 'cr', k = 6) + s(prev_surp, bs = 'cr', k = 6) + te(freq, len, bs = 'cr') + te(prev_freq, prev_len, bs = 'cr'), data = df, weights = weights)
terms_to_predict = c("s(surp)", "s(prev_surp)")
newdata = data.frame(surp=mean_predictors$surp, prev_surp=seq(0,20,by=0.1),
freq=mean_predictors$freq, prev_freq=mean_predictors$freq,
len=mean_predictors$len, prev_len=mean_predictors$len)
# Returns a matrix N_samples * N_terms.
per_term_predictions = predict(m, newdata=newdata, terms=terms_to_predict, type="terms")
# Additive model -- sum across predictor response contributions (matrix columns).
predictions = rowSums(per_term_predictions)
return(newdata %>% mutate(y=predictions))
}
fit_gam_2 = function(df, mean_predictors, alpha=0.05) {
# Bootstrap-resample data
boot_models = df %>% bootstraps(times=10) %>%
# Fit a GAM and get predictions for each sample
mutate(smoothed=map(splits, fit_gam_inner_2, mean_predictors=mean_predictors))
# Extract mean and 5% and 95% percentile y-values for each surprisal value
result = boot_models %>%
unnest(smoothed) %>%
dplyr::select(prev_surp, y) %>%
group_by(prev_surp) %>%
summarise(y_lower=quantile(y, alpha / 2),
y_upper=quantile(y, 1 - alpha / 2),
y=mean(y)) %>%
ungroup()
return (result)
}
gam_modeling_df_2 = provo_df %>%
spread(measure, value) %>%
# mutate(len = nchar(word)) %>% # len has already exists, but do not count punct into len.
group_by(metric, text_id) %>%
arrange(word_text_idx) %>%
mutate(prev_surp = lag(surp),
prev_freq = lag(freq),
prev_len = lag(len),
prev_eyetr_value = lag(eyetr_value)) %>%
ungroup() %>%
drop_na() %>%
rename(psychometric = motr_value)
smooths_df = data.frame()
metrics = c("gaze_duration", "total_duration", "go_past_time", "first_duration")
for (m in metrics) {
print(paste0("Fitting model for ", m))
dummy_df = gam_modeling_df_2 %>% filter(metric == m)
mean_predictors = dummy_df %>% summarise(surp = mean(surp), len = mean(len), freq = mean(freq))
smooths = dummy_df %>% fit_gam_2(., mean_predictors)
#Fix 0 surprisal = 0 ms
gam_smooths = smooths %>% mutate(delta = 0 - y[1], y=y + delta, y_lower= y_lower + delta, y_upper=y_upper + delta)
smooths_df = rbind(smooths_df, gam_smooths %>% mutate(psychometric = m))
}
[1] "Fitting model for gaze_duration"
[1] "Fitting model for total_duration"
[1] "Fitting model for go_past_time"
[1] "Fitting model for first_duration"
get_d_points = function(df) {
x = density(df$surp)$x
y = density(df$surp)$y
return(data.frame(x, y))
}
density_data = data.frame()
for(m in c("gaze_duration", "total_duration", "go_past_time", "first_duration")) {
dummy_df = provo_df %>% filter(metric == m) %>%
do({get_d_points(.)}) %>%
filter(x>0, x<20)
density_data = rbind(density_data, dummy_df %>% mutate(metric=m))
}
# Surprisal curves
ggplot() +
# Density Data
annotate("rect", xmin=0, xmax=20, ymin=-20,ymax=-10, fill="#f4f4f4", color="grey", alpha=1, size = 0) +
geom_line(data = density_data, aes(x=x, y=y*50 - 18), color="#aaaaaa", size = 0.4) +
# Surrp / Rt data
geom_line(data = smooths_df, aes(x=prev_surp, y=y, color = psychometric), size=0.7) +
geom_ribbon(data = smooths_df, aes(x=prev_surp, ymin=y_lower, ymax=y_upper, fill = psychometric), alpha=0.3, size=0.5) +
scale_x_continuous(labels=c(0, 10, 20), breaks=c(0, 10, 20), minor_breaks = NULL) +
facet_wrap(psychometric~., nrow = 1) +
ylab("Slowdown due to Surprisal (ms)") +
xlab("Surprisal of Word") +
ggtitle("MoTR Times and Previous Word Surprisal")

theme(
legend.position = "none",
panel.grid.minor = element_blank()
)
List of 2
$ legend.position : chr "none"
$ panel.grid.minor: list()
..- attr(*, "class")= chr [1:2] "element_blank" "element"
- attr(*, "class")= chr [1:2] "theme" "gg"
- attr(*, "complete")= logi FALSE
- attr(*, "validate")= logi TRUE
Precision and Recall for FPReg
FPReg_df = provo_df %>% filter(metric == "FPReg") %>% spread(measure, value)
confusion_matrix <- table(FPReg_df$motr_value > 0, FPReg_df$eyetr_value > 0)
confusion_matrix
FALSE TRUE
FALSE 79 1518
TRUE 22 918
true_positives <- confusion_matrix[2, 2]
false_positives <- confusion_matrix[2, 1]
false_negatives <- confusion_matrix[1, 2]
precision <- true_positives / (true_positives + false_positives)
recall <- true_positives / (true_positives + false_negatives)
print("precision of Motr FPReg:")
[1] "precision of Motr FPReg:"
print(precision)
[1] 0.9766
print("Recall of Motr FPReg:")
[1] "Recall of Motr FPReg:"
print(recall)
[1] 0.3768
LS0tCnRpdGxlOiAiRXhwbG9yYXRvcnkgQW5hbHlzaXMgZm9yIE1vVFIgUmVhZGluZyBEYXRhIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpgYGB7cn0Kc2hoaCA8LSBzdXBwcmVzc1BhY2thZ2VTdGFydHVwTWVzc2FnZXMgIyBJdCdzIGEgbGlicmFyeSwgc28gc2hoaCEKCnNoaGgobGlicmFyeSggbWdjdiApKQpzaGhoKGxpYnJhcnkoZHBseXIpKQpzaGhoKGxpYnJhcnkoZ2dwbG90MikpCnNoaGgobGlicmFyeShsbWU0KSkKc2hoaChsaWJyYXJ5KHRpZHltdikpCnNoaGgobGlicmFyeShnYW1sc3MpKQpzaGhoKGxpYnJhcnkoZ3N1YmZuKSkKc2hoaChsaWJyYXJ5KGxtZXJUZXN0KSkKc2hoaChsaWJyYXJ5KHRpZHl2ZXJzZSkpCnNoaGgobGlicmFyeShib290KSkKc2hoaChsaWJyYXJ5KHJzYW1wbGUpKQpzaGhoKGxpYnJhcnkocGxvdHJpeCkpCnNoaGgobGlicmFyeShnZ3JlcGVsKSkKc2hoaChsaWJyYXJ5KG1nY3YpKQoKdGhlbWVfc2V0KHRoZW1lX2J3KCkpCm9wdGlvbnMoZGlnaXRzPTQpCm9wdGlvbnMoc2NpcGVuPTk5OSkKc2V0LnNlZWQoNDQ0KQpwaXBlX21lc3NhZ2UgPSBmdW5jdGlvbiguZGF0YSwgc3RhdHVzKSB7bWVzc2FnZShzdGF0dXMpOyAuZGF0YX0KCmBgYAoKCiMgUmVhZCBpbiBNb1RSIERhdGEKCmBgYHtyfQoKcmF0ZSA9IDE2MAoKZmlsZV9wcmVmaXggPSAiL1VzZXJzL2N1aS9EZXNrdG9wL21vdHJfcHJvdm9fMTYwLyIKZm5hbWVzID0gbGlzdC5maWxlcyhwYXRoPWZpbGVfcHJlZml4KQoKZGYgPSBkYXRhLmZyYW1lKCkKZm9yIChmIGluIGZuYW1lcykgewogIHRlbXAgPSByZWFkLmNzdihwYXN0ZTAoZmlsZV9wcmVmaXgsICIvIiwgZikpICU+JQogICAgbXV0YXRlKHN1YmogPSBzdHJfcmVtb3ZlKGYsICJfcmVhZGluZ19tZWFzdXJlcy5jc3YiKSkgJT4lCiAgICByZW5hbWUoZ29fcGFzdF90aW1lID0gZ29fcGFzc190aW1lKQogIGRmID0gcmJpbmQoZGYsIHRlbXApCn0KCmZpbHRlcl9kZiA9IGRmICU+JQogIGdyb3VwX2J5KHBhcmFfbnIsIHN1YmopICU+JQogICAgc3VtbWFyaXNlKGNvcnJlY3QgPSBpZl9lbHNlKHVuaXF1ZShjb3JyZWN0bmVzcykgPT0gMSwgMSwgMCkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICBkcm9wX25hKCkgJT4lCiAgZ3JvdXBfYnkoc3ViaikgJT4lCiAgICBzdW1tYXJpc2UocF9jb3JyZWN0ID0gbWVhbihjb3JyZWN0KSkgJT4lCiAgdW5ncm91cCgpICU+JQogIG11dGF0ZShwX2NvcnJlY3QgPSByb3VuZChwX2NvcnJlY3QsIGRpZ2l0cyA9IDIpKQoKZmlsdGVyX2RmID0gZmlsdGVyX2RmICU+JQogIGZpbHRlcihwX2NvcnJlY3QgPCAwLjgpCiMgVmlldyhmaWx0ZXJfZGYpCiMjIHJlYWRlcl8zOjAuNzAsIHJlYWRlcl82MDowLjc5LCByZWFkZXJfNzY6MC43MiAsIHJlYWRlcl8yNTY6MC43MSAsIHJlYWRlcl8yNjI6MC41NyAKCnJhd19kZiA9IGRmICU+JQogIG11dGF0ZSh3b3JkID0gc3RyX3RyaW0od29yZCkpICU+JQogIG11dGF0ZShzdWJqID0gc3RyX3JlbW92ZShzdWJqLCAicmVhZGVyXyIpKSAlPiUKICBtdXRhdGUoc3ViaiA9IGFzLmludGVnZXIoc3ViaikpICU+JQogIGZpbHRlcighIHN1YmogJWluJSBjKDMsIDYwLCA3NiwgMjU2LCAyNjIpKSAlPiUKICAjIFNlZSBiZWxvdyBmb3IgZmlsdGVyaW5nIG91dCByZWFkaW5nIG1lYXN1cmVzIHRoYXQgYXJlIHN1cGVyIGxvbmcKICBkcGx5cjo6c2VsZWN0KGV4cHJfaWQsIGNvbmRfaWQsIHBhcmFfbnIsIHdvcmQsIHdvcmRfbnIsIGZpcnN0X2R1cmF0aW9uLCB0b3RhbF9kdXJhdGlvbiwgZ2F6ZV9kdXJhdGlvbiwgZ29fcGFzdF90aW1lLCBGUFJlZywgc3ViaikgJT4lCiAgZHJvcF9uYSgpCgpyYXdfZGYKYGBgCgoKYGBge3J9CiMgQXZlcmFnZSBhY3Jvc3Mgc3ViamVjdHMKbW90cl9hZ2dfZGYgPSByYXdfZGYgJT4lCiAgZ2F0aGVyKG1ldHJpYywgdmFsdWUsIDY6MTApICU+JQogICAgZ3JvdXBfYnkocGFyYV9uciwgd29yZF9uciwgd29yZCwgbWV0cmljKSAlPiUKICAgICMgZm9yIG5vbi1iaW5hcnkgbWVhc3VyZXMsIGZpbHRlciBvdXQgdGhhdCBhcmUgc3VwZXIgbG9uZwogICAgbXV0YXRlKG91dGxpZXIgPSBpZl9lbHNlKG1ldHJpYyAhPSAiRlBSZWciICYgdmFsdWUgPiAobWVhbih2YWx1ZSkgKyAzICogc2QodmFsdWUpICksIFQsIEYpKSAlPiUKICAgIGZpbHRlcihvdXRsaWVyID09IEYpICU+JQogICMgIyBGaWx0ZXIgb3V0IHdvcmRzIHdpdGggYSByZWFkaW5nLXRpbWUgb2YgemVybwogICMgbXV0YXRlKHplcm8gPSBpZl9lbHNlKG1ldHJpYyAhPSAiRlBSZWciICYgdmFsdWUgPT0gMCwgVCwgRikpICU+JQogICMgZmlsdGVyKHplcm8gPT0gRikgJT4lCiAgZHJvcF9uYSgpICU+JQogICAgc3VtbWFyaXNlKHZhbHVlID0gbWVhbih2YWx1ZSksCiAgICAgICAgICAgICAgbnN1YmogPSBsZW5ndGgodW5pcXVlKHN1YmopKSkgJT4lCiAgdW5ncm91cCgpICU+JQogIGFycmFuZ2UocGFyYV9uciwgd29yZF9ucikgJT4lCiAgcmVuYW1lKAogICAgdGV4dF9pZCA9IHBhcmFfbnIsCiAgICB3b3JkX3RleHRfaWR4ID0gd29yZF9uciwKICAgIG1vdHJfdmFsdWUgPSB2YWx1ZQogICkKCm1vdHJfYWdnX2RmCgpgYGAKCgoKCiMgQ29tcGFyaXNvbiB0byBQcm92bwoKCmBgYHtyfQojIFJlYWQgaW4gUHJvdm8gc3VycHJpc2FsLCBmcmVxdWVuY3kgYW5kIGxlbmd0aCBkYXRhCnByb3ZvX21vZGVsaW5nX2RmID0gcmVhZC5jc3YoIi9Vc2Vycy9jdWkvRG9jdW1lbnRzL0VUSC9Nb1RSL3BpcGVsaW5lL2FuY2lsbGFyeV9kYXRhL3Byb3ZvX2RmLmNzdiIpICU+JQogIGRwbHlyOjpzZWxlY3QodGV4dF9pZCwgc2VudF9pZCwgdHJpZ2dlcl9pZHgsIHdvcmQsIGZyZXEsIHN1cnAsIGxlbikgJT4lCiAgcmVuYW1lKHdvcmRfaWR4ID0gdHJpZ2dlcl9pZHgpCgpwcm92b19tb2RlbGluZ19kZgoKYGBgCgpgYGB7cn0KIyBSZWFkIGluIFByb3ZvIGV5ZXRyYWNraW5nIGRhdGEKCnByb3ZvX3Jhd19kZiA9IHJlYWQuY3N2KCIvVXNlcnMvY3VpL0RvY3VtZW50cy9FVEgvTW9UUi9waXBlbGluZS9hbmNpbGxhcnlfZGF0YS9wcm92b19leWV0cmFja2luZy5jc3YiKQoKcHJvdm9fZXlldHJhY2tpbmdfZGYgPSBwcm92b19yYXdfZGYgJT4lCiAgZHBseXI6OnNlbGVjdChQYXJ0aWNpcGFudF9JRCwgVGV4dF9JRCwgU2VudGVuY2VfTnVtYmVyLCBXb3JkX0luX1NlbnRlbmNlX051bWJlciwgV29yZCwgV29yZF9OdW1iZXIsIElBX0ZJUlNUX0ZJWF9QUk9HUkVTU0lWRSwgSUFfRklSU1RfUlVOX0RXRUxMX1RJTUUsIElBX0RXRUxMX1RJTUUsIElBX1JFR1JFU1NJT05fUEFUSF9EVVJBVElPTiwgSUFfUkVHUkVTU0lPTl9PVVQpICU+JQogIHJlbmFtZSggI2ZpcnN0X2R1cmF0aW9uID0gSUFfRklSU1RfRklYQVRJT05fRFVSQVRJT04sICAgCiAgICAgICAgICBnYXplX2R1cmF0aW9uID0gSUFfRklSU1RfUlVOX0RXRUxMX1RJTUUsCiAgICAgICAgICB0b3RhbF9kdXJhdGlvbiA9IElBX0RXRUxMX1RJTUUsCiAgICAgICAgICBnb19wYXN0X3RpbWUgPSBJQV9SRUdSRVNTSU9OX1BBVEhfRFVSQVRJT04sCiAgICAgICAgICBzdWJqID0gUGFydGljaXBhbnRfSUQsCiAgICAgICAgICB0ZXh0X2lkID0gVGV4dF9JRCwKICAgICAgICAgIHNlbnRfaWQgPSBTZW50ZW5jZV9OdW1iZXIsCiAgICAgICAgICB3b3JkX2lkeCA9IFdvcmRfSW5fU2VudGVuY2VfTnVtYmVyLAogICAgICAgICAgd29yZF90ZXh0X2lkeCA9IFdvcmRfTnVtYmVyLCAgICMgSUFfSUQ/CiAgICAgICAgICB3b3JkID0gV29yZCwgICAgICAKICAgICAgICAgIEZQUmVnID0gSUFfUkVHUkVTU0lPTl9PVVQsCiAgICAgICAgICBmZl9wcm9ncmVzc2l2ZSA9IElBX0ZJUlNUX0ZJWF9QUk9HUkVTU0lWRSkgJT4lCiAgbXV0YXRlKGZpcnN0X2R1cmF0aW9uID0gZ2F6ZV9kdXJhdGlvbikgJT4lCiAgbXV0YXRlKGdhemVfZHVyYXRpb24gPSBpZmVsc2UoZmZfcHJvZ3Jlc3NpdmUgPT0gMCwgMCwgZ2F6ZV9kdXJhdGlvbiksCiAgICAgICAgIGdvX3Bhc3RfdGltZSA9IGlmZWxzZShmZl9wcm9ncmVzc2l2ZSA9PSAwLCAwLCBnb19wYXN0X3RpbWUpKSAlPiUKICBkcGx5cjo6c2VsZWN0KC1mZl9wcm9ncmVzc2l2ZSkgJT4lCiAgZ2F0aGVyKG1ldHJpYywgdmFsdWUsIDc6MTEpICU+JQogIG11dGF0ZSh2YWx1ZSA9IGlmX2Vsc2UoaXMubmEodmFsdWUpLCBhcy5pbnRlZ2VyKDApLCBhcy5pbnRlZ2VyKHZhbHVlKSkpICU+JQogIGRyb3BfbmEoKSAlPiUKICBtdXRhdGUod29yZCA9IHN0cl90cmltKHdvcmQpKSAlPiUKICBtdXRhdGUoc3ViaiA9IHN0cl9yZW1vdmUoc3ViaiwgIlN1YiIpKSAlPiUKICBtdXRhdGUoc3ViaiA9IGFzLmludGVnZXIoc3ViaikpICU+JQogICAgZ3JvdXBfYnkodGV4dF9pZCwgd29yZF90ZXh0X2lkeCwgc2VudF9pZCwgd29yZF9pZHgsIHdvcmQsIG1ldHJpYykgJT4lCiAgICBtdXRhdGUob3V0bGllciA9IGlmX2Vsc2UobWV0cmljICE9ICJGUFJlZyIgJiB2YWx1ZSA+IChtZWFuKHZhbHVlKSArIDMgKiBzZCh2YWx1ZSkgKSwgVCwgRikpICU+JQogICAgZmlsdGVyKG91dGxpZXIgPT0gRikgJT4lCiAgdW5ncm91cCgpICMlPiUKICAjICMgRmlsdGVyIG91dCB3b3JkcyB3aXRoIGEgcmVhZGluZy10aW1lIG9mIHplcm8KICAjIG11dGF0ZSh6ZXJvID0gaWZfZWxzZShtZXRyaWMgIT0gIkZQUmVnIiAmIHZhbHVlID09IDAsVCwgRikpICU+JQogICMgZmlsdGVyKHplcm8gPT0gRikKCiMgQWdncmVnYXRlIGNyb3NzLXBhcnRpY2lwYW50IGRhdGEgZm9yIGFsbCBzdWJqZWN0cwpwcm92b19leWV0cmFja2luZ19hZ2dfZGYgPSBwcm92b19leWV0cmFja2luZ19kZiAlPiUKICBncm91cF9ieSh0ZXh0X2lkLCB3b3JkX3RleHRfaWR4LCBzZW50X2lkLCB3b3JkX2lkeCwgd29yZCwgbWV0cmljKSAlPiUKICAgIHN1bW1hcmlzZSh2YWx1ZSA9IG1lYW4odmFsdWUpLCAuZ3JvdXBzID0gJ2Ryb3AnKSAKCgpgYGAKCmBgYHtyfQoKIyBTcGxpdCB0aGUgZXlldHJhY2tpbmcgZGF0YSBpbiB0d28gYnkgc3ViamVjdHMgdG8gc2VlIGhvdyB3ZWxsIGl0IGNvcnJlbGF0ZXMgd2l0aCBpdHNlbGYKcHJvdm9fZXlldHJhY2tpbmdfc3ViajFfZGYgPSBwcm92b19leWV0cmFja2luZ19kZiAlPiUKICBmaWx0ZXIoc3ViaiA8PSA0MikgJT4lCiAgZ3JvdXBfYnkodGV4dF9pZCwgd29yZF90ZXh0X2lkeCwgc2VudF9pZCwgd29yZF9pZHgsIHdvcmQsIG1ldHJpYykgJT4lCiAgICBzdW1tYXJpc2UodmFsdWUgPSBtZWFuKHZhbHVlKSkgJT4lCiAgdW5ncm91cCgpICU+JQogIHJlbmFtZSh2YWx1ZV8xID0gdmFsdWUpICU+JQogIGRwbHlyOjpzZWxlY3QoLXNlbnRfaWQsIC13b3JkX2lkeCkKICAKICBwcm92b19leWV0cmFja2luZ19zdWJqMl9kZiA9IHByb3ZvX2V5ZXRyYWNraW5nX2RmICU+JQogIGZpbHRlcihzdWJqID4gNDIpICU+JQogIGdyb3VwX2J5KHRleHRfaWQsIHdvcmRfdGV4dF9pZHgsIHNlbnRfaWQsIHdvcmRfaWR4LCB3b3JkLCBtZXRyaWMpICU+JQogICAgc3VtbWFyaXNlKHZhbHVlID0gbWVhbih2YWx1ZSkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAgIHJlbmFtZSh2YWx1ZV8yID0gdmFsdWUpJT4lCiAgZHBseXI6OnNlbGVjdCgtc2VudF9pZCwgLXdvcmRfaWR4KQogIApwcm92b19leWV0cl9ncm91cGVkX2RmID0gbWVyZ2UocHJvdm9fZXlldHJhY2tpbmdfc3ViajJfZGYsIHByb3ZvX2V5ZXRyYWNraW5nX3N1YmoxX2RmLCBieT1jKCJ0ZXh0X2lkIiwgIndvcmRfdGV4dF9pZHgiLCAibWV0cmljIikpICU+JQogIGZpbHRlcih3b3JkLnggPT0gd29yZC55KSAlPiUKICBkcGx5cjo6c2VsZWN0KC13b3JkLngsIC13b3JkLnkpICU+JQogIGdyb3VwX2J5KG1ldHJpYykgJT4lCiAgICBtdXRhdGUobW90cl9vdXRsaWVyID0gaWZfZWxzZShtZXRyaWMgIT0gIkZQUmVnIiAmIHZhbHVlXzEgPiAobWVhbih2YWx1ZV8xKSArIDMgKiBzZCh2YWx1ZV8xKSApLCBULCBGKSkgJT4lCiAgICBmaWx0ZXIobW90cl9vdXRsaWVyID09IEYpICU+JQogICAgbXV0YXRlKGV5ZXRyX291dGxpZXIgPSBpZl9lbHNlKG1ldHJpYyAhPSAiRlBSZWciICYgdmFsdWVfMiA+IChtZWFuKHZhbHVlXzIpICsgMyAqIHNkKHZhbHVlXzIpICksIFQsIEYpKSAlPiUKICAgIGZpbHRlcihleWV0cl9vdXRsaWVyID09IEYpICU+JQogIHVuZ3JvdXAoKSAlPiUKICBnYXRoZXIobWVhc3VyZSwgdmFsdWUsIGMoInZhbHVlXzEiLCAidmFsdWVfMiIpKSAlPiUKICBkcGx5cjo6c2VsZWN0KC1tb3RyX291dGxpZXIsIC1leWV0cl9vdXRsaWVyKQoKCmBgYAoKCmBgYHtyfQpwcm92b19kZiA9IG1lcmdlKHByb3ZvX2V5ZXRyYWNraW5nX2FnZ19kZiwgcHJvdm9fbW9kZWxpbmdfZGYsIGJ5PWMoInRleHRfaWQiLCAic2VudF9pZCIsICJ3b3JkX2lkeCIpKSAlPiUKICBtdXRhdGUod29yZF90ZXh0X2lkeCA9IGFzLmludGVnZXIod29yZF90ZXh0X2lkeCAtIDEpKSAlPiUKICBhcnJhbmdlKHRleHRfaWQsIHNlbnRfaWQsIHdvcmRfaWR4KSAlPiUKICByZW5hbWUoZXlldHJfdmFsdWUgPSB2YWx1ZSkgCgpwcm92b19kZiA9IG1lcmdlKHByb3ZvX2RmLCBtb3RyX2FnZ19kZiwgYnk9YygidGV4dF9pZCIsICJ3b3JkX3RleHRfaWR4IiwgIm1ldHJpYyIpKSAlPiUKYXJyYW5nZSh0ZXh0X2lkLCBzZW50X2lkLCB3b3JkX2lkeCkgJT4lCiAgIyBhbG1vc3QgYWxsIHRoZSB3b3JkLnggIT0gd29yZC55IGlzIGJlY2F1c2Ugb2Ygbm9ybWFsaXphdGlvbiBwcm9ibGVtLCBzbyB3ZSBjYW4ga2VlcCB0aGVtLCBpbnN0ZWFkLCBkZWxldGluZyBzb21lIHNwZWNpYWwgY2FzZXMKICBmaWx0ZXIoISh0ZXh0X2lkID09IDEzICYgd29yZF90ZXh0X2lkeCA+PSAyMCAmIHdvcmRfdGV4dF9pZHggPD0gNTIpKSAlPiUKICBmaWx0ZXIoISh0ZXh0X2lkID09IDMgJiB3b3JkX3RleHRfaWR4ID49IDQ2ICYgd29yZF90ZXh0X2lkeCA8PSA1NykpICU+JQojICMgZmlsdGVyKHdvcmQueSA9PSB3b3JkKSAlPiUKZHBseXI6OnNlbGVjdCgtd29yZC54LCAtd29yZC55KSAlPiUKZ3JvdXBfYnkobWV0cmljKSAlPiUKICBtdXRhdGUobW90cl9vdXRsaWVyID0gaWZfZWxzZShtZXRyaWMgIT0gIkZQUmVnIiAmIG1vdHJfdmFsdWUgPiAobWVhbihtb3RyX3ZhbHVlKSArIDMgKiBzZChtb3RyX3ZhbHVlKSApLCBULCBGKSkgJT4lCiAgZmlsdGVyKG1vdHJfb3V0bGllciA9PSBGKSAlPiUKICBtdXRhdGUoZXlldHJfb3V0bGllciA9IGlmX2Vsc2UobWV0cmljICE9ICJGUFJlZyIgJiBleWV0cl92YWx1ZSA+IChtZWFuKGV5ZXRyX3ZhbHVlKSArIDMgKiBzZChleWV0cl92YWx1ZSkgKSwgVCwgRikpICU+JQogIGZpbHRlcihleWV0cl9vdXRsaWVyID09IEYpICU+JQp1bmdyb3VwKCkgJT4lCmdhdGhlcihtZWFzdXJlLCB2YWx1ZSwgYygiZXlldHJfdmFsdWUiLCAibW90cl92YWx1ZSIpKSAlPiUKICBkcGx5cjo6c2VsZWN0KC1tb3RyX291dGxpZXIsIC1leWV0cl9vdXRsaWVyKQoKCmBgYAoKYGBge3J9CnByb3ZvX2RmICU+JQogIG11dGF0ZShtZWFzdXJlID0gaWZfZWxzZShtZWFzdXJlID09ICJleWV0cl92YWx1ZSIsICJFeWV0cmFja2luZyBWYWx1ZSIsICJNb1RSIFZhbHVlIikpICU+JQogIGZpbHRlcihtZXRyaWMgIT0gIkZQUmVnIikgJT4lCiAgZ2dwbG90KGFlcyh4ID0gdmFsdWUsIGNvbG9yPW1ldHJpYykpICsKICAgIGdlb21fZGVuc2l0eSgpICsKICAgIGZhY2V0X3dyYXAoLn5tZWFzdXJlLCBzY2FsZXM9ImZyZWVfeSIpICsKICAgIHhsYWIoIlJlYWRpbmcgVGltZSAobXMpIikKCiMgZ2dzYXZlKCIuLi92aXN1YWxpemF0aW9uL2RlbnNpdHkucG5nIiwgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNiwgaGVpZ2h0ID0gMi41KQpgYGAKCgpgYGB7cn0KcHJpbnQoIkdhemUgRHVyYXRpb24iKQpjb3JfZGYgPSBwcm92b19kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSAiZ2F6ZV9kdXJhdGlvbiIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRleWV0cl92YWx1ZSwgY29yX2RmJG1vdHJfdmFsdWUpJGVzdGltYXRlKQoKY29yX2RmID0gcHJvdm9fZXlldHJfZ3JvdXBlZF9kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSAiZ2F6ZV9kdXJhdGlvbiIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGNvcl9kZiR2YWx1ZV8xLCBjb3JfZGYkdmFsdWVfMikkZXN0aW1hdGUpCgpwcmludCgiRmlyc3QgRHVyYXRpb24iKQpnZF9kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09ICJmaXJzdF9kdXJhdGlvbiIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGdkX2RmJGV5ZXRyX3ZhbHVlLCBnZF9kZiRtb3RyX3ZhbHVlKSRlc3RpbWF0ZSkKCmNvcl9kZiA9IHByb3ZvX2V5ZXRyX2dyb3VwZWRfZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gImZpcnN0X2R1cmF0aW9uIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJHZhbHVlXzEsIGNvcl9kZiR2YWx1ZV8yKSRlc3RpbWF0ZSkKCnByaW50KCJHbyBQYXN0IFRpbWUiKQoKZ2RfZGYgPSBwcm92b19kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSAiZ29fcGFzdF90aW1lIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoZ2RfZGYkZXlldHJfdmFsdWUsIGdkX2RmJG1vdHJfdmFsdWUpJGVzdGltYXRlKQoKY29yX2RmID0gcHJvdm9fZXlldHJfZ3JvdXBlZF9kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSAiZ29fcGFzdF90aW1lIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJHZhbHVlXzEsIGNvcl9kZiR2YWx1ZV8yKSRlc3RpbWF0ZSkKCnByaW50KCJUb3RhbCBEdXJhdGlvbiIpCgpnZF9kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09ICJ0b3RhbF9kdXJhdGlvbiIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGdkX2RmJGV5ZXRyX3ZhbHVlLCBnZF9kZiRtb3RyX3ZhbHVlKSRlc3RpbWF0ZSkKCmNvcl9kZiA9IHByb3ZvX2V5ZXRyX2dyb3VwZWRfZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gInRvdGFsX2R1cmF0aW9uIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJHZhbHVlXzEsIGNvcl9kZiR2YWx1ZV8yKSRlc3RpbWF0ZSkKCnByaW50KCJSZWdyZXNzaW9uIikKCnJlZ19kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09ICJGUFJlZyIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KHJlZ19kZiRleWV0cl92YWx1ZSwgcmVnX2RmJG1vdHJfdmFsdWUpJGVzdGltYXRlKQoKY29yX2RmID0gcHJvdm9fZXlldHJfZ3JvdXBlZF9kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSAiRlBSZWciKSAlPiUgZ3JvdXBfYnkodGV4dF9pZCwgbWV0cmljLCBtZWFzdXJlKSAlPiUKICBzdW1tYXJpemUodmFsdWUgPSBtZWFuKHZhbHVlLCBuYS5ybSA9IFRSVUUpLCAuZ3JvdXBzID0gJ2Ryb3AnKSAlPiUgc3ByZWFkKG1lYXN1cmUsIHZhbHVlKQpwcmludChjb3IudGVzdChjb3JfZGYkdmFsdWVfMSwgY29yX2RmJHZhbHVlXzIpJGVzdGltYXRlKQoKYGBgCgoKCmBgYHtyfQoKcHJvdm9fZGYgJT4lCiAgZmlsdGVyKG1ldHJpYyAhPSAiRlBSZWciKSAlPiUKICBzcHJlYWQobWVhc3VyZSwgdmFsdWUpICU+JQogIGdncGxvdChhZXMoeCA9IG1vdHJfdmFsdWUsIHk9ZXlldHJfdmFsdWUpKSArCiAgICBnZW9tX3BvaW50KGFscGhhID0gMC4wNSkgKwogICAgZ2VvbV9hYmxpbmUoc2xvcGU9MSwgaW50ZXJjZXB0PTAsIGNvbG9yID0gImJsYWNrIikgKwogICAgI3N0YXRfc3VtbWFyeV9iaW4oYmlucz0xMDAsIGZ1bi5kYXRhID0gIm1lYW5fY2xfYm9vdCIsIHNpemUgPSAwLjA1KSArCiAgICBmYWNldF93cmFwKC5+bWV0cmljLCBzY2FsZXMgPSAiZnJlZSIsIG5yb3cgPSAxKSArCiAgICBjb29yZF9jYXJ0ZXNpYW4oeWxpbT1jKDAsIDUwMCksIHhsaW09YygwLCA1MDApKSArCiAgICBnZW9tX3Ntb290aCgpCgoKIyBnZ3NhdmUoIi4uL3Zpc3VhbGl6YXRpb24vbWV0cmljX2Nvci5wbmciLCBkZXZpY2UgPSAicG5nIiwgd2lkdGggPSA2LCBoZWlnaHQgPSAyLjUpCmBgYAojIyBDb3JyZWxhdGlvbnMgdG8gV29yZC1MZXZlbCBTdGF0aXN0aWNhbCBQcm9wZXJ0aWVzCgpgYGB7cn0KcHJpbnQoIkdhemUgRHVyYXRpb24iKQpwcmludCgiTGVuIikKY29yX2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gImdhemVfZHVyYXRpb24iKSAlPiUgc3ByZWFkKG1lYXN1cmUsIHZhbHVlKQpwcmludChjb3IudGVzdChjb3JfZGYkZXlldHJfdmFsdWUsIGNvcl9kZiRsZW4pJGVzdGltYXRlKQpwcmludChjb3IudGVzdChjb3JfZGYkbW90cl92YWx1ZSwgY29yX2RmJGxlbikkZXN0aW1hdGUpCgpwcmludCgiRnJlcSIpCmNvcl9kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09ICJnYXplX2R1cmF0aW9uIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJGV5ZXRyX3ZhbHVlLCBjb3JfZGYkZnJlcSkkZXN0aW1hdGUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRtb3RyX3ZhbHVlLCBjb3JfZGYkZnJlcSkkZXN0aW1hdGUpCgpwcmludCgiU3VycCIpCmNvcl9kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09ICJnYXplX2R1cmF0aW9uIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJGV5ZXRyX3ZhbHVlLCBjb3JfZGYkc3VycCkkZXN0aW1hdGUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRtb3RyX3ZhbHVlLCBjb3JfZGYkc3VycCkkZXN0aW1hdGUpCgoKYGBgCgpgYGB7cn0KcHJpbnQoIlRvdGFsIER1cmF0aW9uIikKcHJpbnQoIkxlbiIpCmNvcl9kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09ICJ0b3RhbF9kdXJhdGlvbiIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRleWV0cl92YWx1ZSwgY29yX2RmJGxlbikkZXN0aW1hdGUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRtb3RyX3ZhbHVlLCBjb3JfZGYkbGVuKSRlc3RpbWF0ZSkKCnByaW50KCJGcmVxIikKY29yX2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gInRvdGFsX2R1cmF0aW9uIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJGV5ZXRyX3ZhbHVlLCBjb3JfZGYkZnJlcSkkZXN0aW1hdGUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRtb3RyX3ZhbHVlLCBjb3JfZGYkZnJlcSkkZXN0aW1hdGUpCgpwcmludCgiU3VycCIpCmNvcl9kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09ICJ0b3RhbF9kdXJhdGlvbiIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRleWV0cl92YWx1ZSwgY29yX2RmJHN1cnApJGVzdGltYXRlKQpwcmludChjb3IudGVzdChjb3JfZGYkbW90cl92YWx1ZSwgY29yX2RmJHN1cnApJGVzdGltYXRlKQpgYGAKCmBgYHtyfQpwcmludCgiRmlyc3QgRHVyYXRpb24iKQpwcmludCgiTGVuIikKY29yX2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gImZpcnN0X2R1cmF0aW9uIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJGV5ZXRyX3ZhbHVlLCBjb3JfZGYkbGVuKSRlc3RpbWF0ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJG1vdHJfdmFsdWUsIGNvcl9kZiRsZW4pJGVzdGltYXRlKQoKcHJpbnQoIkZyZXEiKQpjb3JfZGYgPSBwcm92b19kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSAiZmlyc3RfZHVyYXRpb24iKSAlPiUgc3ByZWFkKG1lYXN1cmUsIHZhbHVlKQpwcmludChjb3IudGVzdChjb3JfZGYkZXlldHJfdmFsdWUsIGNvcl9kZiRmcmVxKSRlc3RpbWF0ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJG1vdHJfdmFsdWUsIGNvcl9kZiRmcmVxKSRlc3RpbWF0ZSkKCnByaW50KCJTdXJwIikKY29yX2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gImZpcnN0X2R1cmF0aW9uIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJGV5ZXRyX3ZhbHVlLCBjb3JfZGYkc3VycCkkZXN0aW1hdGUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRtb3RyX3ZhbHVlLCBjb3JfZGYkc3VycCkkZXN0aW1hdGUpCmBgYAoKYGBge3J9CnByaW50KCJHbyBQYXN0IFRpbWUiKQpwcmludCgiTGVuIikKY29yX2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gImdvX3Bhc3RfdGltZSIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRleWV0cl92YWx1ZSwgY29yX2RmJGxlbikkZXN0aW1hdGUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRtb3RyX3ZhbHVlLCBjb3JfZGYkbGVuKSRlc3RpbWF0ZSkKCnByaW50KCJGcmVxIikKY29yX2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gImdvX3Bhc3RfdGltZSIpICU+JSBzcHJlYWQobWVhc3VyZSwgdmFsdWUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRleWV0cl92YWx1ZSwgY29yX2RmJGZyZXEpJGVzdGltYXRlKQpwcmludChjb3IudGVzdChjb3JfZGYkbW90cl92YWx1ZSwgY29yX2RmJGZyZXEpJGVzdGltYXRlKQoKcHJpbnQoIlN1cnAiKQpjb3JfZGYgPSBwcm92b19kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSAiZ29fcGFzdF90aW1lIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKcHJpbnQoY29yLnRlc3QoY29yX2RmJGV5ZXRyX3ZhbHVlLCBjb3JfZGYkc3VycCkkZXN0aW1hdGUpCnByaW50KGNvci50ZXN0KGNvcl9kZiRtb3RyX3ZhbHVlLCBjb3JfZGYkc3VycCkkZXN0aW1hdGUpCmBgYAoKYGBge3J9CnByb3ZvX2RmICU+JQogIGdhdGhlcih3b3JkX3Byb3AsIHdvcmRfcHJvcF92YWwsIGMoImZyZXEiLCAibGVuIiwgInN1cnAiKSkgJT4lCiAgbXV0YXRlKG1lYXN1cmUgPSBpZl9lbHNlKG1lYXN1cmUgPT0gImV5ZXRyX3ZhbHVlIiwgIkV5ZXRyYWNraW5nIFZhbHVlIiwgIk1vVFIgVmFsdWUiKSkgJT4lCiAgbXV0YXRlKHdvcmRfcHJvcCA9IGNhc2Vfd2hlbigKICAgIHdvcmRfcHJvcCA9PSAiZnJlcSIgfiAiRnJlcXVlbmN5IiwKICAgIHdvcmRfcHJvcCA9PSAibGVuIiB+ICJMZW5ndGgiLAogICAgd29yZF9wcm9wID09ICJzdXJwIiB+ICJTdXJwcmlzYWwiCiAgKSkgJT4lCiAgZmlsdGVyKG1ldHJpYyA9PSAiZ2F6ZV9kdXJhdGlvbiIpICU+JQogIGdncGxvdChhZXMoeCA9IHZhbHVlLCB5PXdvcmRfcHJvcF92YWwsIGNvbG9yID0gbWVhc3VyZSkpICsKICAgIGdlb21fcG9pbnQoYWxwaGEgPSAwLjEpICsKICAgIGZhY2V0X3dyYXAobWVhc3VyZX53b3JkX3Byb3AsIHNjYWxlcz0iZnJlZSIsIHN0cmlwLnBvc2l0aW9uID0gInJpZ2h0IikgKwogICAgZ2VvbV9zbW9vdGgoY29sb3IgPSAiZ3JleSIpICsKICAgIHhsYWIoIlJlYWRpbmcgTWVhc3VyZSIpICsKICB0aGVtZSgKICAgIGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIiwKICAgIHN0cmlwLnBsYWNlbWVudCA9ICJvdXRzaWRlIgogICkKCiMgZ2dzYXZlKCIuLi92aXN1YWxpemF0aW9uL3dvcmRfcHJvcF9jb21wcy5wbmciLCBkZXZpY2UgPSAicG5nIiwgd2lkdGggPSA2LCBoZWlnaHQgPSAzKQoKYGBgCgpgYGB7cn0KcHJvdm9fZGYgJT4lCiAgZ2dwbG90KGFlcyh4ID0gdmFsdWUsIHk9ZnJlcSwgY29sb3I9bWV0cmljKSkgKwogICAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMSkgKwogICAgZmFjZXRfZ3JpZChtZXRyaWN+bWVhc3VyZSwgc2NhbGVzPSJmcmVlIikgKwogICAgZ2VvbV9zbW9vdGgoKQpgYGAKCmBgYHtyfQpwcm92b19kZiAlPiUKICBnZ3Bsb3QoYWVzKHggPSB2YWx1ZSwgeT1zdXJwLCBjb2xvcj1tZXRyaWMpKSArCiAgICBnZW9tX3BvaW50KGFscGhhID0gMC4yKSArCiAgICBmYWNldF9ncmlkKG1ldHJpY35tZWFzdXJlLCBzY2FsZXM9ImZyZWUiKSArCiAgICBnZW9tX3Ntb290aCgpCmBgYAoKCiMjIFNoYXBlIG9mIHN1cnByaXNhbCAvIFJUIHJlbGF0aW9uc2hpcAoKIyMjIGZvciBjdXJyZW50IHdvcmQ6CmBgYHtyfQoKZml0X2dhbV9pbm5lciA9IGZ1bmN0aW9uKGJvb3RzdHJhcF9zYW1wbGUsIG1lYW5fcHJlZGljdG9ycykgewogIAogIGRmID0gYm9vdHN0cmFwX3NhbXBsZSRkYXRhCiAgd2VpZ2h0cyA9IHRhYnVsYXRlKGFzLmludGVnZXIoYm9vdHN0cmFwX3NhbXBsZSksIG5yb3coZGYpKQogIAogIG0gPSBnYW0ocHN5Y2hvbWV0cmljIH4gcyhzdXJwLCBicyA9ICdjcicsIGsgPSA2KSArIHMocHJldl9zdXJwLCBicyA9ICdjcicsIGsgPSA2KSArIHRlKGZyZXEsIGxlbiwgYnMgPSAnY3InKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gJ2NyJyksIGRhdGEgPSBkZiwgd2VpZ2h0cyA9IHdlaWdodHMpCiAgdGVybXNfdG9fcHJlZGljdCA9IGMoInMoc3VycCkiLCAicyhwcmV2X3N1cnApIikKICAKICBuZXdkYXRhID0gZGF0YS5mcmFtZShzdXJwPXNlcSgwLDIwLGJ5PTAuMSksIHByZXZfc3VycD1tZWFuX3ByZWRpY3RvcnMkc3VycCwKICAgICAgICAgICAgICAgICAgICAgICAjc3VycD1tZWFuX3ByZWRpY3RvcnMkc3VycCwgcHJldl9zdXJwPXNlcSgwLDIwLGJ5PTAuMSksCiAgICAgICAgICAgICAgICAgICAgICAgZnJlcT1tZWFuX3ByZWRpY3RvcnMkZnJlcSwgcHJldl9mcmVxPW1lYW5fcHJlZGljdG9ycyRmcmVxLAogICAgICAgICAgICAgICAgICAgICAgIGxlbj1tZWFuX3ByZWRpY3RvcnMkbGVuLCBwcmV2X2xlbj1tZWFuX3ByZWRpY3RvcnMkbGVuKQoKICAjIFJldHVybnMgYSBtYXRyaXggTl9zYW1wbGVzICogTl90ZXJtcy4KICBwZXJfdGVybV9wcmVkaWN0aW9ucyA9IHByZWRpY3QobSwgbmV3ZGF0YT1uZXdkYXRhLCB0ZXJtcz10ZXJtc190b19wcmVkaWN0LCB0eXBlPSJ0ZXJtcyIpCgogICMgQWRkaXRpdmUgbW9kZWwgLS0gc3VtIGFjcm9zcyBwcmVkaWN0b3IgcmVzcG9uc2UgY29udHJpYnV0aW9ucyAobWF0cml4IGNvbHVtbnMpLgogIHByZWRpY3Rpb25zID0gcm93U3VtcyhwZXJfdGVybV9wcmVkaWN0aW9ucykKCiAgcmV0dXJuKG5ld2RhdGEgJT4lIG11dGF0ZSh5PXByZWRpY3Rpb25zKSkKfQoKZml0X2dhbSA9IGZ1bmN0aW9uKGRmLCBtZWFuX3ByZWRpY3RvcnMsIGFscGhhPTAuMDUpIHsKICAjIEJvb3RzdHJhcC1yZXNhbXBsZSBkYXRhCiAgYm9vdF9tb2RlbHMgPSBkZiAlPiUgYm9vdHN0cmFwcyh0aW1lcz0xMCkgJT4lIAogICAjIEZpdCBhIEdBTSBhbmQgZ2V0IHByZWRpY3Rpb25zIGZvciBlYWNoIHNhbXBsZQogICAgbXV0YXRlKHNtb290aGVkPW1hcChzcGxpdHMsIGZpdF9nYW1faW5uZXIsIG1lYW5fcHJlZGljdG9ycz1tZWFuX3ByZWRpY3RvcnMpKQogIAogICMgRXh0cmFjdCBtZWFuIGFuZCA1JSBhbmQgOTUlIHBlcmNlbnRpbGUgeS12YWx1ZXMgZm9yIGVhY2ggc3VycHJpc2FsIHZhbHVlCiAgcmVzdWx0ID0gYm9vdF9tb2RlbHMgJT4lIAogICAgdW5uZXN0KHNtb290aGVkKSAlPiUgCiAgICBkcGx5cjo6c2VsZWN0KHN1cnAsIHkpICU+JSAKICAgICNkcGx5cjo6c2VsZWN0KHByZXZfc3VycCwgeSkgJT4lIAogICAgZ3JvdXBfYnkoc3VycCkgJT4lIAogICAgI2dyb3VwX2J5KHByZXZfc3VycCkgJT4lCiAgICAgIHN1bW1hcmlzZSh5X2xvd2VyPXF1YW50aWxlKHksIGFscGhhIC8gMiksIAogICAgICAgICAgICAgICAgeV91cHBlcj1xdWFudGlsZSh5LCAxIC0gYWxwaGEgLyAyKSwKICAgICAgICAgICAgICAgIHk9bWVhbih5KSkgJT4lIAogICAgdW5ncm91cCgpCiAgCiAgcmV0dXJuIChyZXN1bHQpCn0KCmBgYAoKCmBgYHtyfQoKZ2FtX21vZGVsaW5nX2RmID0gcHJvdm9fZGYgJT4lCiAgc3ByZWFkKG1lYXN1cmUsIHZhbHVlKSAlPiUKICAjIG11dGF0ZShsZW4gPSBuY2hhcih3b3JkKSkgJT4lICAgICAgIyBsZW4gaGFzIGFscmVhZHkgZXhpc3RzLCBidXQgZG8gbm90IGNvdW50IHB1bmN0IGludG8gbGVuLgogIGdyb3VwX2J5KG1ldHJpYywgdGV4dF9pZCkgJT4lCiAgICBhcnJhbmdlKHdvcmRfdGV4dF9pZHgpICU+JQogICAgbXV0YXRlKHByZXZfc3VycCA9IGxhZyhzdXJwKSwKICAgICAgICAgICBwcmV2X2ZyZXEgPSBsYWcoZnJlcSksCiAgICAgICAgICAgcHJldl9sZW4gPSBsYWcobGVuKSwKICAgICAgICAgICBwcmV2X2V5ZXRyX3ZhbHVlID0gbGFnKGV5ZXRyX3ZhbHVlKSkgJT4lCiAgdW5ncm91cCgpICU+JQogIGRyb3BfbmEoKSAlPiUKICByZW5hbWUocHN5Y2hvbWV0cmljID0gbW90cl92YWx1ZSkKCnNtb290aHNfZGYgPSBkYXRhLmZyYW1lKCkKCm1ldHJpY3MgPSBjKCJnYXplX2R1cmF0aW9uIiwgInRvdGFsX2R1cmF0aW9uIiwgImdvX3Bhc3RfdGltZSIsICJmaXJzdF9kdXJhdGlvbiIpCmZvciAobSBpbiBtZXRyaWNzKSB7CiAgcHJpbnQocGFzdGUwKCJGaXR0aW5nIG1vZGVsIGZvciAiLCBtKSkKICBkdW1teV9kZiA9IGdhbV9tb2RlbGluZ19kZiAlPiUgZmlsdGVyKG1ldHJpYyA9PSBtKQogIG1lYW5fcHJlZGljdG9ycyA9IGR1bW15X2RmICU+JSBzdW1tYXJpc2Uoc3VycCA9IG1lYW4oc3VycCksIGxlbiA9IG1lYW4obGVuKSwgZnJlcSA9IG1lYW4oZnJlcSkpCiAgc21vb3RocyA9IGR1bW15X2RmICU+JSBmaXRfZ2FtKC4sIG1lYW5fcHJlZGljdG9ycykKICAjRml4IDAgc3VycHJpc2FsID0gMCBtcwogIGdhbV9zbW9vdGhzID0gc21vb3RocyAlPiUgbXV0YXRlKGRlbHRhID0gMCAtIHlbMV0sIHk9eSArIGRlbHRhLCB5X2xvd2VyPSB5X2xvd2VyICsgZGVsdGEsIHlfdXBwZXI9eV91cHBlciArIGRlbHRhKQogIHNtb290aHNfZGYgPSByYmluZChzbW9vdGhzX2RmLCBnYW1fc21vb3RocyAlPiUgbXV0YXRlKHBzeWNob21ldHJpYyA9IG0pKQp9CgpgYGAKCiMjIyBHZXQgRGVuc2l0eSBEYXRhCgpgYGB7cn0KZ2V0X2RfcG9pbnRzID0gZnVuY3Rpb24oZGYpIHsKICAgIHggPSBkZW5zaXR5KGRmJHN1cnApJHgKICAgIHkgPSBkZW5zaXR5KGRmJHN1cnApJHkKICAgIHJldHVybihkYXRhLmZyYW1lKHgsIHkpKQogIH0KCmRlbnNpdHlfZGF0YSA9IGRhdGEuZnJhbWUoKQoKZm9yKG0gaW4gYygiZ2F6ZV9kdXJhdGlvbiIsICJ0b3RhbF9kdXJhdGlvbiIsICJnb19wYXN0X3RpbWUiLCAiZmlyc3RfZHVyYXRpb24iKSkgewogIGR1bW15X2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gbSkgJT4lCiAgICAgIGRvKHtnZXRfZF9wb2ludHMoLil9KSAlPiUKICAgICAgZmlsdGVyKHg+MCwgeDwyMCkKICBkZW5zaXR5X2RhdGEgPSByYmluZChkZW5zaXR5X2RhdGEsIGR1bW15X2RmICU+JSBtdXRhdGUobWV0cmljPW0pKQp9CgpgYGAKCgpgYGB7cn0KCiMgU3VycHJpc2FsIGN1cnZlcwogIGdncGxvdCgpICsKICAgICAgIyBEZW5zaXR5IERhdGEKICAgICAgYW5ub3RhdGUoInJlY3QiLCB4bWluPTAsIHhtYXg9MjAsIHltaW49LTIwLHltYXg9LTEwLCBmaWxsPSIjZjRmNGY0IiwgY29sb3I9ImdyZXkiLCBhbHBoYT0xLCBzaXplID0gMCkgKwogICAgICBnZW9tX2xpbmUoZGF0YSA9IGRlbnNpdHlfZGF0YSwgYWVzKHg9eCwgeT15KjUwIC0gMTgpLCBjb2xvcj0iI2FhYWFhYSIsIHNpemUgPSAwLjQpICsKICAgICAgIyBTdXJycCAvIFJ0IGRhdGEKICAgICAgI2dlb21fbGluZShkYXRhID0gc21vb3Roc19kZiwgYWVzKHg9cHJldl9zdXJwLCB5PXksIGNvbG9yID0gcHN5Y2hvbWV0cmljKSwgc2l6ZT0wLjcpICsKICAgICAgZ2VvbV9saW5lKGRhdGEgPSBzbW9vdGhzX2RmLCBhZXMoeD1zdXJwLCB5PXksIGNvbG9yID0gcHN5Y2hvbWV0cmljKSwgc2l6ZT0wLjcpICsKICAgICAgI2dlb21fcmliYm9uKGRhdGEgPSBzbW9vdGhzX2RmLCBhZXMoeD1wcmV2X3N1cnAsIHltaW49eV9sb3dlciwgeW1heD15X3VwcGVyLCBmaWxsID0gcHN5Y2hvbWV0cmljKSwgYWxwaGE9MC4zLCBzaXplPTAuNSkgKwogICAgICBnZW9tX3JpYmJvbihkYXRhID0gc21vb3Roc19kZiwgYWVzKHg9c3VycCwgeW1pbj15X2xvd2VyLCB5bWF4PXlfdXBwZXIsIGZpbGwgPSBwc3ljaG9tZXRyaWMpLCBhbHBoYT0wLjMsIHNpemU9MC41KSArCiAgICAgIHNjYWxlX3hfY29udGludW91cyhsYWJlbHM9YygwLCAxMCwgMjApLCBicmVha3M9YygwLCAxMCwgMjApLCBtaW5vcl9icmVha3MgPSBOVUxMKSArCiAgICAgIGZhY2V0X3dyYXAocHN5Y2hvbWV0cmljfi4sIG5yb3cgPSAxKSArCiAgICAgIHlsYWIoIlNsb3dkb3duIGR1ZSB0byBTdXJwcmlzYWwgKG1zKSIpICsKICAgICAgeGxhYigiU3VycHJpc2FsIG9mIFdvcmQiKSArCiAgICAgIGdndGl0bGUoIk1vVFIgVGltZXMgYW5kIEN1cnJlbnQgV29yZCBTdXJwcmlzYWwiKQogIHRoZW1lKAogICAgbGVnZW5kLnBvc2l0aW9uID0gIm5vbmUiLAogICAgcGFuZWwuZ3JpZC5taW5vciA9IGVsZW1lbnRfYmxhbmsoKQogICkKICAKIyBnZ3NhdmUoIi4uL3Zpc3VhbGl6YXRpb24vc3VycHJpc2FsX3J0X2xpbmsucG5nIiwgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNiwgaGVpZ2h0ID0gMi41KQoKCmBgYAoKIyMjIGZvciBwcmV2aW91cyB3b3JkOgoKYGBge3J9CmZpdF9nYW1faW5uZXJfMiA9IGZ1bmN0aW9uKGJvb3RzdHJhcF9zYW1wbGUsIG1lYW5fcHJlZGljdG9ycykgewogIAogIGRmID0gYm9vdHN0cmFwX3NhbXBsZSRkYXRhCiAgd2VpZ2h0cyA9IHRhYnVsYXRlKGFzLmludGVnZXIoYm9vdHN0cmFwX3NhbXBsZSksIG5yb3coZGYpKQogIAogIG0gPSBnYW0ocHN5Y2hvbWV0cmljIH4gcyhzdXJwLCBicyA9ICdjcicsIGsgPSA2KSArIHMocHJldl9zdXJwLCBicyA9ICdjcicsIGsgPSA2KSArIHRlKGZyZXEsIGxlbiwgYnMgPSAnY3InKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gJ2NyJyksIGRhdGEgPSBkZiwgd2VpZ2h0cyA9IHdlaWdodHMpCiAgdGVybXNfdG9fcHJlZGljdCA9IGMoInMoc3VycCkiLCAicyhwcmV2X3N1cnApIikKICAKICBuZXdkYXRhID0gZGF0YS5mcmFtZShzdXJwPW1lYW5fcHJlZGljdG9ycyRzdXJwLCBwcmV2X3N1cnA9c2VxKDAsMjAsYnk9MC4xKSwKICAgICAgICAgICAgICAgICAgICAgICBmcmVxPW1lYW5fcHJlZGljdG9ycyRmcmVxLCBwcmV2X2ZyZXE9bWVhbl9wcmVkaWN0b3JzJGZyZXEsCiAgICAgICAgICAgICAgICAgICAgICAgbGVuPW1lYW5fcHJlZGljdG9ycyRsZW4sIHByZXZfbGVuPW1lYW5fcHJlZGljdG9ycyRsZW4pCgogICMgUmV0dXJucyBhIG1hdHJpeCBOX3NhbXBsZXMgKiBOX3Rlcm1zLgogIHBlcl90ZXJtX3ByZWRpY3Rpb25zID0gcHJlZGljdChtLCBuZXdkYXRhPW5ld2RhdGEsIHRlcm1zPXRlcm1zX3RvX3ByZWRpY3QsIHR5cGU9InRlcm1zIikKCiAgIyBBZGRpdGl2ZSBtb2RlbCAtLSBzdW0gYWNyb3NzIHByZWRpY3RvciByZXNwb25zZSBjb250cmlidXRpb25zIChtYXRyaXggY29sdW1ucykuCiAgcHJlZGljdGlvbnMgPSByb3dTdW1zKHBlcl90ZXJtX3ByZWRpY3Rpb25zKQoKICByZXR1cm4obmV3ZGF0YSAlPiUgbXV0YXRlKHk9cHJlZGljdGlvbnMpKQp9CgpmaXRfZ2FtXzIgPSBmdW5jdGlvbihkZiwgbWVhbl9wcmVkaWN0b3JzLCBhbHBoYT0wLjA1KSB7CiAgIyBCb290c3RyYXAtcmVzYW1wbGUgZGF0YQogIGJvb3RfbW9kZWxzID0gZGYgJT4lIGJvb3RzdHJhcHModGltZXM9MTApICU+JSAKICAgIyBGaXQgYSBHQU0gYW5kIGdldCBwcmVkaWN0aW9ucyBmb3IgZWFjaCBzYW1wbGUKICAgIG11dGF0ZShzbW9vdGhlZD1tYXAoc3BsaXRzLCBmaXRfZ2FtX2lubmVyXzIsIG1lYW5fcHJlZGljdG9ycz1tZWFuX3ByZWRpY3RvcnMpKQogIAogICMgRXh0cmFjdCBtZWFuIGFuZCA1JSBhbmQgOTUlIHBlcmNlbnRpbGUgeS12YWx1ZXMgZm9yIGVhY2ggc3VycHJpc2FsIHZhbHVlCiAgcmVzdWx0ID0gYm9vdF9tb2RlbHMgJT4lIAogICAgdW5uZXN0KHNtb290aGVkKSAlPiUgCiAgICBkcGx5cjo6c2VsZWN0KHByZXZfc3VycCwgeSkgJT4lCiAgICBncm91cF9ieShwcmV2X3N1cnApICU+JQogICAgICBzdW1tYXJpc2UoeV9sb3dlcj1xdWFudGlsZSh5LCBhbHBoYSAvIDIpLCAKICAgICAgICAgICAgICAgIHlfdXBwZXI9cXVhbnRpbGUoeSwgMSAtIGFscGhhIC8gMiksCiAgICAgICAgICAgICAgICB5PW1lYW4oeSkpICU+JSAKICAgIHVuZ3JvdXAoKQogIAogIHJldHVybiAocmVzdWx0KQp9CmBgYAoKYGBge3J9CmdhbV9tb2RlbGluZ19kZl8yID0gcHJvdm9fZGYgJT4lCiAgc3ByZWFkKG1lYXN1cmUsIHZhbHVlKSAlPiUKICAjIG11dGF0ZShsZW4gPSBuY2hhcih3b3JkKSkgJT4lICAjIGxlbiBoYXMgYWxyZWFkeSBleGlzdHMsIGJ1dCBkbyBub3QgY291bnQgcHVuY3QgaW50byBsZW4uCiAgZ3JvdXBfYnkobWV0cmljLCB0ZXh0X2lkKSAlPiUKICAgIGFycmFuZ2Uod29yZF90ZXh0X2lkeCkgJT4lCiAgICBtdXRhdGUocHJldl9zdXJwID0gbGFnKHN1cnApLAogICAgICAgICAgIHByZXZfZnJlcSA9IGxhZyhmcmVxKSwKICAgICAgICAgICBwcmV2X2xlbiA9IGxhZyhsZW4pLAogICAgICAgICAgIHByZXZfZXlldHJfdmFsdWUgPSBsYWcoZXlldHJfdmFsdWUpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgZHJvcF9uYSgpICU+JQogIHJlbmFtZShwc3ljaG9tZXRyaWMgPSBtb3RyX3ZhbHVlKQoKc21vb3Roc19kZiA9IGRhdGEuZnJhbWUoKQoKbWV0cmljcyA9IGMoImdhemVfZHVyYXRpb24iLCAidG90YWxfZHVyYXRpb24iLCAiZ29fcGFzdF90aW1lIiwgImZpcnN0X2R1cmF0aW9uIikKZm9yIChtIGluIG1ldHJpY3MpIHsKICBwcmludChwYXN0ZTAoIkZpdHRpbmcgbW9kZWwgZm9yICIsIG0pKQogIGR1bW15X2RmID0gZ2FtX21vZGVsaW5nX2RmXzIgJT4lIGZpbHRlcihtZXRyaWMgPT0gbSkKICBtZWFuX3ByZWRpY3RvcnMgPSBkdW1teV9kZiAlPiUgc3VtbWFyaXNlKHN1cnAgPSBtZWFuKHN1cnApLCBsZW4gPSBtZWFuKGxlbiksIGZyZXEgPSBtZWFuKGZyZXEpKQogIHNtb290aHMgPSBkdW1teV9kZiAlPiUgZml0X2dhbV8yKC4sIG1lYW5fcHJlZGljdG9ycykKICAjRml4IDAgc3VycHJpc2FsID0gMCBtcwogIGdhbV9zbW9vdGhzID0gc21vb3RocyAlPiUgbXV0YXRlKGRlbHRhID0gMCAtIHlbMV0sIHk9eSArIGRlbHRhLCB5X2xvd2VyPSB5X2xvd2VyICsgZGVsdGEsIHlfdXBwZXI9eV91cHBlciArIGRlbHRhKQogIHNtb290aHNfZGYgPSByYmluZChzbW9vdGhzX2RmLCBnYW1fc21vb3RocyAlPiUgbXV0YXRlKHBzeWNob21ldHJpYyA9IG0pKQp9CmBgYAoKYGBge3J9CmdldF9kX3BvaW50cyA9IGZ1bmN0aW9uKGRmKSB7CiAgICB4ID0gZGVuc2l0eShkZiRzdXJwKSR4CiAgICB5ID0gZGVuc2l0eShkZiRzdXJwKSR5CiAgICByZXR1cm4oZGF0YS5mcmFtZSh4LCB5KSkKICB9CgpkZW5zaXR5X2RhdGEgPSBkYXRhLmZyYW1lKCkKCmZvcihtIGluIGMoImdhemVfZHVyYXRpb24iLCAidG90YWxfZHVyYXRpb24iLCAiZ29fcGFzdF90aW1lIiwgImZpcnN0X2R1cmF0aW9uIikpIHsKICBkdW1teV9kZiA9IHByb3ZvX2RmICU+JSBmaWx0ZXIobWV0cmljID09IG0pICU+JQogICAgICBkbyh7Z2V0X2RfcG9pbnRzKC4pfSkgJT4lCiAgICAgIGZpbHRlcih4PjAsIHg8MjApCiAgZGVuc2l0eV9kYXRhID0gcmJpbmQoZGVuc2l0eV9kYXRhLCBkdW1teV9kZiAlPiUgbXV0YXRlKG1ldHJpYz1tKSkKfQpgYGAKCgpgYGB7cn0KIyBTdXJwcmlzYWwgY3VydmVzCiAgZ2dwbG90KCkgKwogICAgICAjIERlbnNpdHkgRGF0YQogICAgICBhbm5vdGF0ZSgicmVjdCIsIHhtaW49MCwgeG1heD0yMCwgeW1pbj0tMjAseW1heD0tMTAsIGZpbGw9IiNmNGY0ZjQiLCBjb2xvcj0iZ3JleSIsIGFscGhhPTEsIHNpemUgPSAwKSArCiAgICAgIGdlb21fbGluZShkYXRhID0gZGVuc2l0eV9kYXRhLCBhZXMoeD14LCB5PXkqNTAgLSAxOCksIGNvbG9yPSIjYWFhYWFhIiwgc2l6ZSA9IDAuNCkgKwogICAgICAjIFN1cnJwIC8gUnQgZGF0YQogICAgICBnZW9tX2xpbmUoZGF0YSA9IHNtb290aHNfZGYsIGFlcyh4PXByZXZfc3VycCwgeT15LCBjb2xvciA9IHBzeWNob21ldHJpYyksIHNpemU9MC43KSArCiAgICAgIGdlb21fcmliYm9uKGRhdGEgPSBzbW9vdGhzX2RmLCBhZXMoeD1wcmV2X3N1cnAsIHltaW49eV9sb3dlciwgeW1heD15X3VwcGVyLCBmaWxsID0gcHN5Y2hvbWV0cmljKSwgYWxwaGE9MC4zLCBzaXplPTAuNSkgKwogICAgICBzY2FsZV94X2NvbnRpbnVvdXMobGFiZWxzPWMoMCwgMTAsIDIwKSwgYnJlYWtzPWMoMCwgMTAsIDIwKSwgbWlub3JfYnJlYWtzID0gTlVMTCkgKwogICAgICBmYWNldF93cmFwKHBzeWNob21ldHJpY34uLCBucm93ID0gMSkgKwogICAgICB5bGFiKCJTbG93ZG93biBkdWUgdG8gU3VycHJpc2FsIChtcykiKSArCiAgICAgIHhsYWIoIlN1cnByaXNhbCBvZiBXb3JkIikgKwogICAgICBnZ3RpdGxlKCJNb1RSIFRpbWVzIGFuZCBQcmV2aW91cyBXb3JkIFN1cnByaXNhbCIpCiAgdGhlbWUoCiAgICBsZWdlbmQucG9zaXRpb24gPSAibm9uZSIsCiAgICBwYW5lbC5ncmlkLm1pbm9yID0gZWxlbWVudF9ibGFuaygpCiAgKQpgYGAKCiMjIFByZWNpc2lvbiBhbmQgUmVjYWxsIGZvciBGUFJlZwoKYGBge3J9CkZQUmVnX2RmID0gcHJvdm9fZGYgJT4lIGZpbHRlcihtZXRyaWMgPT0gIkZQUmVnIikgJT4lIHNwcmVhZChtZWFzdXJlLCB2YWx1ZSkKY29uZnVzaW9uX21hdHJpeCA8LSB0YWJsZShGUFJlZ19kZiRtb3RyX3ZhbHVlID4gMCwgRlBSZWdfZGYkZXlldHJfdmFsdWUgPiAwKQpjb25mdXNpb25fbWF0cml4Cgp0cnVlX3Bvc2l0aXZlcyA8LSBjb25mdXNpb25fbWF0cml4WzIsIDJdCmZhbHNlX3Bvc2l0aXZlcyA8LSBjb25mdXNpb25fbWF0cml4WzIsIDFdCmZhbHNlX25lZ2F0aXZlcyA8LSBjb25mdXNpb25fbWF0cml4WzEsIDJdCgpwcmVjaXNpb24gPC0gdHJ1ZV9wb3NpdGl2ZXMgLyAodHJ1ZV9wb3NpdGl2ZXMgKyBmYWxzZV9wb3NpdGl2ZXMpCnJlY2FsbCA8LSB0cnVlX3Bvc2l0aXZlcyAvICh0cnVlX3Bvc2l0aXZlcyArIGZhbHNlX25lZ2F0aXZlcykKCnByaW50KCJwcmVjaXNpb24gb2YgTW90ciBGUFJlZzoiKQpwcmludChwcmVjaXNpb24pCnByaW50KCJSZWNhbGwgb2YgTW90ciBGUFJlZzoiKQpwcmludChyZWNhbGwpCmBgYAoKCgo=